###############
# Plot population means
###############
def plot_means(df, labels):
# Normalize the data
df_norm = ((df - df.min()) / (df.max() - df.min()))
# Get the mean of the normalized data
means = df_norm.mean()
# Add the labels to the normalized data
df_norm['label'] = labels
# Group the data by label
grouped = df_norm.groupby('label')
# Get the mean of each group
means_by_label = grouped.mean()
# Set up the plot
fig, ax = plt.subplots(dpi=150)
# Plot the overall mean as a scatterplot
ax.scatter(means.values, means.index, marker='o', label='Population Mean', c='#000000')
# Iterate through each group and plot the mean as a scatterplot
colors = ['blue', 'orange', 'green', 'red', 'purple', 'brown', 'gray', 'black', 'pink', 'yellow', '#ff00ff', '#bbf90f', '#00ffff', '#c79fef']
color_count = 0
for i, (label, row) in enumerate(means_by_label.iterrows()):
ax.scatter(row.values, row.index, marker='x', label=label, c=colors[i])
color_count += 1
# Set the y-axis tick labels to be the same as the mean data
ax.set_yticklabels(means.index)
# Add a legend to the plot
ax.legend(bbox_to_anchor=(1,1), loc="upper left")
# Show the plot
plt.show()
###############
# Correlation Heatmap
###############
def heatmap(dataframe, treshold=.05):
'''plots correlation heatmap'''
# calculate corelation matrix
corr = dataframe.corr()
f, ax = plt.subplots(figsize=(13,10))
sns.heatmap(corr,cmap='Spectral',fmt='.2f')
# this loop only labels fields in the heatmap abough a specific treshold (default .05)
for i in range(len(corr)):
for j in range(len(corr)):
if abs(corr.iloc[i, j]) > treshold:
ax.text(j+0.5, i+0.5, '{:.2f}'.format(corr.iloc[i, j]),
ha='center', va='center', color='black')
###############
# Visualizing Component Planes (Self-Organizing-Maps)
###############
def plot_component_planes(weights, # Array of weights
features, # List of feature names
M=3, N=4, # Number of rows and columns
figsize=(20,20),
figlayout=(3,4),
title="Component Planes",
cmap=cm.magma
):
# create grids xx, yy
xx, yy = np.meshgrid(np.arange(M), np.arange(N))
xx = xx.astype(float)
yy = yy.astype(float)
xx[::-2] -= 0.5
xx = xx.T
yy = yy.T
weights_ = weights.reshape((M,N,len(features)))
fig = plt.figure(figsize=figsize, constrained_layout=True)
subfigs = fig.subfigures(figlayout[0], figlayout[1], wspace=.15)
## Normalize color scale to range of all values
colornorm = mpl_colors.Normalize(vmin=np.min(weights),
vmax=np.max(weights))
for cpi, sf in zip(range(len(features)), subfigs.flatten()):
sf.suptitle(features[cpi], y=0.95)
axs = sf.subplots(1,1, )
axs.set_aspect('equal')
## Normalize color scale to range of values in each component
colornorm = mpl_colors.Normalize(vmin=np.min(weights_[:,:,cpi]),
vmax=np.max(weights_[:,:,cpi]))
# iteratively add hexagons
for i in range(weights_.shape[0]):
for j in range(weights_.shape[1]):
wy = yy[(i, j)] * np.sqrt(3) / 2
hexagon = RegularPolygon((xx[(i, j)], wy),
numVertices=6,
radius=.99 / np.sqrt(3),
facecolor=cmap(colornorm(weights_[i, j, cpi])),
alpha=1,
linewidth=.5,
edgecolor=cmap(colornorm(weights_[i, j, cpi]))
)
axs.add_patch(hexagon)
# only run this block if matplotlib >= 3.6.x
mplv = [int(i) for i in mplver.split('.')]
if mplv[1] >= 6:
# Add colorbar
divider = make_axes_locatable(axs)
ax_cb = divider.append_axes("right", size="7%")#, pad="2%")
# Create a Mappable object
cmap_sm = plt.cm.ScalarMappable(cmap=cmap, norm=colornorm)
cmap_sm.set_array([])
# Create custom colorbar
cb1 = colorbar.Colorbar(ax_cb,
orientation='vertical',
alpha=1,
mappable=cmap_sm
)
cb1.ax.get_yaxis().labelpad = 16
# Add colorbar to plot
sf.add_axes(ax_cb)
# Remove axes for hex plot
axs.margins(.05)
axs.axis("off")
fig.suptitle(title)
plt.show()
plt.rcParams['figure.dpi'] = 72
###############
# Plot Clustering Performances
###############
def plot_scores(array_lists, df_list, label_lists):
'''
Function to plot r-squared and silhouette scores of the different clustering solutions
Output is a barchart visualization comparing the different cluster solutions
'''
rows = len(array_lists)
fig, axs = plt.subplots(rows, 2, figsize=(12, 5 * rows), dpi=150)
plt.subplots_adjust(wspace=0.3)
row_labels = ["All Features", "Perspective - Value", "Perspective- Demography", "DBSCAN ALL", "DBSCAN Value Perspective"]
for i in range(rows):
arrays = array_lists[i]
df = df_list[i]
labels = label_lists[i]
# create lists to store scores
r2_scores = []
silhouette_scores = []
# loop over each list of labels
for j, array in enumerate(arrays):
# calculate scores with "def r2" (see below)
r2_score = r2(df, array)
silhouette = silhouette_score(df,array)
r2_scores.append(r2_score)
silhouette_scores.append(silhouette)
unique_count = len(np.unique(array))
labels[j] = f"({labels[j]} {unique_count} cluster)"
# add barchart to subplot
ax1 = axs[i][0]
ax1.bar(labels, r2_scores, color = '#bf5252')
ax1.set_ylabel('R-Quadrat-Score')
ax1.set_title(row_labels[i])
ax1.set_xticklabels(labels, rotation=90)
ax1.axhline(max(r2_scores), linestyle='dashed', color='red')
ax2 = axs[i][1]
ax2.bar(labels, silhouette_scores)
ax2.set_ylabel('Silhouette-Score')
ax2.set_xticklabels(labels, rotation=90)
ax2.axhline(max(silhouette_scores), linestyle='dashed', color='red')
fig.tight_layout()
plt.show()
###############
# Plot Elbow Curve K-Prototype
###############
def plot_elbow_curve_Kpro(start, end, data):
'''
Function to plot K-Prototype elbow curve
'''
no_of_clusters = list(range(start, end+1))
print(no_of_clusters)
cost_values = []
# loop over cluster sizes and fit & predict KPrototypes for each cluster size
for k in no_of_clusters:
test_model = KPrototypes(n_clusters=k, init='Huang', random_state=42)
test_model.fit_predict(data, categorical=categorical_index)
print(k,"Clusters Done")
cost_values.append(test_model.cost_)
sns.set_theme(style="whitegrid", palette="bright", font_scale=1.2)
# set up lineplot
plt.figure(figsize=(15, 7))
ax = sns.lineplot(x=no_of_clusters, y=cost_values, marker="o", dashes=False)
ax.set_title('Elbow curve', fontsize=18)
ax.set_xlabel('No of clusters', fontsize=14)
ax.set_ylabel('Cost', fontsize=14)
ax.set(xlim=(start-0.1, end+0.1))
plt.plot()
###############
# Visualization Functions for Cluster Results
###############
def cluster_plots(dataframe):
'''
Function to plot r-squared and silhouette score
Will be used to find the optimal cluster sizes for all methods that include KMeans
'''
range_clusters = range(1, 11)
inertia = []
avg_silhouette = []
# loop over possible cluster sizes
for n_clus in range_clusters:
# calculate r-squared score via inertia_
kmclust = KMeans(n_clusters=n_clus, init='k-means++', n_init=15, random_state=1)
kmclust.fit(dataframe)
inertia.append(kmclust.inertia_)
if n_clus == 1:
continue
# fit and predict for calculating silhouette score
cluster_labels = kmclust.fit_predict(dataframe)
# calculate silhouette score
silhouette_avg = silhouette_score(dataframe, cluster_labels)
avg_silhouette.append(silhouette_avg)
# The inertia and average silhuette plots
f, ax = plt.subplots(1, 2, figsize=(7,3), dpi=200)
ax[0].plot(range_clusters, inertia)
ax[0].set_xticks(range_clusters)
ax[0].set_ylabel("Inertia: SSw", size=10)
ax[0].set_xlabel("Number of clusters", size=10)
ax[0].set_title("Elbow Method", size=10)
ax[1].plot(range_clusters[1:],avg_silhouette)
ax[1].set_xticks(range_clusters)
ax[1].set_ylabel("Average silhouette", size=10)
ax[1].set_xlabel("Number of clusters", size=10)
ax[1].set_title("Average silhouette plot over clusters", size=10)
plt.show()
def r2(features, labels):
'''
Function to calculate the r-squared score
Will be used in the function "plot_scores"
'''
def get_ss(dat):
ss = np.sum(dat.var() * (dat.count() - 1))
return ss
sst = get_ss(features)
df = pd.DataFrame()
df = features.copy()
df['labels'] = labels.tolist()
ssw_labels = df.groupby(by='labels').apply(get_ss)
ssb = sst - np.sum(ssw_labels)
r2 = ssb / sst
return r2
def overwiev_mean(features, labels):
'''
Creates a pandas dataframe with the means of each cluster
'''
feat = features.copy()
feat['labels'] = labels.tolist()
# group by labels and calculate the average/mean
feat = feat.groupby('labels').mean()
return feat
def plot_umap_tsne(dataframe, labels):
'''
Function to calculate and plot UMAP and T-SNE
Both Visualizations are then plotted next to each other for comparison
'''
# make sure that no categorical features are in the input dataframe
metric_features_eval = [i for i in dataframe.columns if not i in ['Education', 'Children']]
# set coloring otherwise clusters have different colors in different visualizations (e.g. Radar Plot)
colors = ['blue', 'orange', 'green', 'red', 'purple', 'brown', 'gray', 'black', 'pink', 'yellow', '#ff00ff', '#bbf90f','#00ffff', '#c79fef']
label_colors = [colors[i] for i in labels]
dataframe = dataframe[metric_features_eval]
#calculate umap and tsne
umap_transformed = umap.UMAP(n_neighbors=20, min_dist=0.25).fit_transform(dataframe)
tsne_transformed = TSNE(n_components=2, perplexity=60).fit_transform(dataframe)
umap_x = umap_transformed[:, 0]
umap_y = umap_transformed[:, 1]
tsne_x = tsne_transformed[:, 0]
tsne_y = tsne_transformed[:, 1]
#create a subplot to be able to visualize both techniques next to each other
fig, (ax1, ax2) = plt.subplots(nrows=1, ncols=2, figsize=(12, 6), dpi=300)
ax1.scatter(umap_x, umap_y, c=label_colors, s=1)
ax1.set_title("UMAP")
ax2.scatter(tsne_x, tsne_y, c=label_colors, s=3)
ax2.set_title("TSNE")
fig.tight_layout()
plt.show()
def pairplot(features, labels):
'''
Returns a seaborn pairplot with hue set to the specific cluster colors
'''
feat = features.copy()
feat = feat[[i for i in feat.columns if not i in ['EducDeg', 'Children']]]
feat['labels'] = labels.tolist()
# set coloring otherwise clusters have different colors in different visualizations (e.g. Radar Plot)
color_palette = {0:'blue', 1:'orange', 2:'green', 3:'red', 4:'purple', 5:'brown', 6:'gray', 7:'black'}
sns.pairplot(feat, hue='labels', palette = color_palette)
plt.show()
def decision_tree_eval(dataframe, labels):
'''
Calculates a crossvalidated decision tree score
Plots tree (max_depth=5)
'''
df_decision_tree = dataframe.copy()
df_decision_tree['Clusters'] = labels
# train test split
x, y = df_decision_tree.drop(columns=['Clusters']), df_decision_tree.Clusters
xtrain, xval, ytrain, yval = train_test_split(x,y)
# calculate cross validated scores
model = DecisionTreeClassifier()
scores = cross_val_score(model, x, y, cv=5)
# fit tree again for visualizing
clf = DecisionTreeClassifier(max_depth=5).fit(xtrain,ytrain)
# plot tree with sklearn "plot_tree" function
fig, axe = plt.subplots(figsize=(60,15),dpi=200)
plot_tree(clf, ax = axe, fontsize=15, feature_names=dataframe.columns, filled=True)
plt.show()
return np.mean(scores)
# ComplexRadar Source:https://towardsdatascience.com/how-to-create-and-visualize-complex-radar-charts-f7764d0f3652
class ComplexRadar():
def __init__(self, fig, variables, ranges, n_ring_levels=5, show_scales=True):
# Calculate angles and create for each variable an axes
# Consider here the trick with having the first axes element twice (len+1)
angles = np.arange(0, 360, 360./len(variables))
axes = [fig.add_axes([0.1,0.1,0.9,0.9], polar=True, label = "axes{}".format(i)) for i in range(len(variables)+1)]
# Ensure clockwise rotation (first variable at the top N)
for ax in axes:
ax.set_theta_zero_location('N')
ax.set_theta_direction(-1)
ax.set_axisbelow(True)
# Writing the ranges on each axes
for i, ax in enumerate(axes):
# Here we do the trick by repeating the first iteration
j = 0 if (i==0 or i==1) else i-1
ax.set_ylim(*ranges[j])
# Set endpoint to True if you like to have values right before the last circle
grid = np.linspace(*ranges[j], num=n_ring_levels,
endpoint=False)
gridlabel = ["{}".format(round(x,2)) for x in grid]
gridlabel[0] = "" # remove values from the center
lines, labels = ax.set_rgrids(grid, labels=gridlabel, angle=angles[j])
ax.set_ylim(*ranges[j])
ax.spines["polar"].set_visible(False)
ax.grid(visible=False)
if show_scales == False:
ax.set_yticklabels([])
# Set all axes except the first one unvisible
for ax in axes[1:]:
ax.patch.set_visible(False)
ax.xaxis.set_visible(False)
# Setting the attributes
self.angle = np.deg2rad(np.r_[angles, angles[0]])
self.ranges = ranges
self.ax = axes[0]
self.ax1 = axes[1]
self.plot_counter = 0
# Draw (inner) circles and lines
self.ax.yaxis.grid()
self.ax.xaxis.grid()
# Draw outer circle
self.ax.spines['polar'].set_visible(True)
# ax1 is the duplicate of axes[0] (self.ax)
# Remove everything from ax1 except the plot itself
self.ax1.axis('off')
self.ax1.set_zorder(9)
# Create the outer labels for each variable
l, text = self.ax.set_thetagrids(angles, labels=variables)
# Beautify them
labels = [t.get_text() for t in self.ax.get_xticklabels()]
labels = ['\n'.join(textwrap.wrap(l, 15,
break_long_words=False)) for l in labels]
self.ax.set_xticklabels(labels)
for t,a in zip(self.ax.get_xticklabels(),angles):
if a == 0:
t.set_ha('center')
elif a > 0 and a < 180:
t.set_ha('left')
elif a == 180:
t.set_ha('center')
else:
t.set_ha('right')
self.ax.tick_params(axis='both', pad=15)
def _scale_data(self, data, ranges):
"""Scales data[1:] to ranges[0]"""
for d, (y1, y2) in zip(data[1:], ranges[1:]):
assert (y1 <= d <= y2) or (y2 <= d <= y1)
x1, x2 = ranges[0]
d = data[0]
sdata = [d]
for d, (y1, y2) in zip(data[1:], ranges[1:]):
sdata.append((d-y1) / (y2-y1) * (x2 - x1) + x1)
return sdata
def plot(self, data, *args, **kwargs):
"""Plots a line"""
sdata = self._scale_data(data, self.ranges)
self.ax1.plot(self.angle, np.r_[sdata, sdata[0]], *args, **kwargs)
self.plot_counter = self.plot_counter+1
def fill(self, data, *args, **kwargs):
"""Plots an area"""
sdata = self._scale_data(data, self.ranges)
self.ax1.fill(self.angle, np.r_[sdata, sdata[0]], *args, **kwargs)
def use_legend(self, *args, **kwargs):
"""Shows a legend"""
self.ax1.legend(*args, **kwargs)
def set_title(self, title, pad=25, **kwargs):
"""Set a title"""
self.ax.set_title(title,pad=pad, **kwargs)
def radar_plot(dataframe, labels):
data = dataframe.copy()
data['cluster'] = labels
result = data.groupby(['cluster']).mean()
min_max_per_variable = data.describe().T[['min', 'max']]
min_max_per_variable['min'] = min_max_per_variable['min'].apply(lambda x: int(x))
min_max_per_variable['max'] = min_max_per_variable['max'].apply(lambda x: math.ceil(x))
variables = result.columns
ranges = list(min_max_per_variable.itertuples(index=False, name=None))
fig1 = plt.figure(figsize=(10, 10))
radar = ComplexRadar(fig1, variables, ranges, show_scales=True)
# set coloring otherwise clusters have different colors in different visualizations (e.g. Radar Plot)
colors = ['blue', 'orange', 'green', 'red', 'purple', 'brown', 'gray', 'black', 'pink', 'yellow', '#ff00ff', '#bbf90f','#00ffff', '#c79fef']
color_counter = 0
# loop over cluster means and draw radar plot line
for g in result.index:
radar.plot(result.loc[g].values, label=f"cluster {g} (n = {len(data[data.cluster==g])})", **{'c':colors[g]})
color_counter += 0
radar.set_title('Radar Chart')
radar.use_legend(loc='lower center', bbox_to_anchor=(0.15, -0.25),ncol=radar.plot_counter)
plt.show()
###############
# Combine Visualizations Functions
###############
def cluster_viz_report(features_all_unscaled ,features_all_scaled, labels, verbose=1):
'''
Function to combine the visualization functions set up above
depending on the input parameter verbose it's more or less detailed
'''
if verbose==1:
print('\n\n--------------------------------------------- CLUSTER PROFILE - MEANS ------------------------------------------\n'),
plot_means(features_all_unscaled, labels)
print('\n\n--------------------------------------------- CLUSTER PROFILE - RADAR ------------------------------------------\n'),
radar_plot(features_all_unscaled, labels)
elif verbose==2:
print('\n\n--------------------------------------------- VALUE OVERVIEW BY MEAN ------------------------------------------\n'),
display(overwiev_mean(features_all_unscaled, labels)),
print('\n\n--------------------------------------------- CLUSTER PROFILE - MEANS ------------------------------------------\n'),
plot_means(features_all_unscaled, labels)
print('\n\n--------------------------------------------- CLUSTER PROFILE - RADAR ------------------------------------------\n'),
radar_plot(features_all_unscaled, labels)
print('\n\n-------------------------------------------------- T-SNE & U-MAP -----------------------------------------------\n'),
try:
plot_umap_tsne(features_all_unscaled,labels),
except:
import umap.umap_ as umap
plot_umap_tsne(features_all_unscaled,labels),
print('\n\n-------------------------------------------------- DECISION TREE -----------------------------------------------\n'),
decision_tree_eval(features_all_unscaled, labels),
elif verbose==3:
print('\n\n--------------------------------------------- VALUE OVERVIEW BY MEAN ------------------------------------------\n'),
display(overwiev_mean(features_all_unscaled, labels)),
print('\n\n--------------------------------------------- CLUSTER PROFILE - MEANS ------------------------------------------\n'),
plot_means(features_all_unscaled, labels)
print('\n\n--------------------------------------------- CLUSTER PROFILE - RADAR ------------------------------------------\n'),
radar_plot(features_all_unscaled, labels),
print('\n\n-------------------------------------------------- T-SNE & U-MAP -----------------------------------------------\n'),
try:
plot_umap_tsne(features_all_unscaled,labels),
except:
import umap.umap_ as umap
plot_umap_tsne(features_all_unscaled,labels),
print('\n\n-------------------------------------------------- DECISION TREE -----------------------------------------------\n'),
decision_tree_eval(features_all_unscaled, labels),
print('\n\n-------------------------------------------CLUSTER DISTRIBUTION BY FEATURE -------------------------------------\n'),
pairplot(features_all_unscaled, labels)
